const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  let totalBlogs = [];
  const url = 'https://github.com/toutiaoio/awesome-architecture';
  const filename = 'github7.json';

  const browser = await puppeteer.launch({headless: false});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  await page.setExtraHTTPHeaders({
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
  });
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  await page.goto(url, {waitUntil: 'domcontentloaded'});
  await page.waitFor(10000);

  let blog1 = await page.$$eval('#readme > div.Box-body.px-5.pb-5 > article > ul > li > a', eles => eles.map(el => {
    return {
      title: el.innerText,
      content: el.href
    }
  }));

  totalBlogs = blog1;
  console.log(totalBlogs.length);

  let writerStream = fs.createWriteStream(`crawler_data/${filename}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');
};


main();


